{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4ec4dc4e",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'NoneType' object is not iterable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-3a66cdb22a60>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m    115\u001b[0m \u001b[1;31m#     url = 'http://www.7k7k.com/flash/197869.htm'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    116\u001b[0m \u001b[1;31m#     info = spider_all_action(web_content(url))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 117\u001b[1;33m     \u001b[0md\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mspider_info\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    118\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'multiThreadxiuxian_intro.txt'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'w'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    119\u001b[0m         \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mensure_ascii\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-12-3a66cdb22a60>\u001b[0m in \u001b[0;36mspider_info\u001b[1;34m(info)\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mspider_info\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[0mdic2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minfo\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;31m#             if\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: 'NoneType' object is not iterable"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n"
     ]
    }
   ],
   "source": [
    "import urllib.request\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "import threading\n",
    "#请求网页页面内容函数\n",
    "def web_content(url):\n",
    "    #请求页面地址\n",
    "    req = urllib.request.Request(url)\n",
    "    # 增加头部信息\n",
    "    req.add_header('User-Agent',\n",
    "                   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36')\n",
    "    # 读取网页内容 编码方式为gbk 此地方有一个坑\n",
    "    html = urllib.request.urlopen(req).read().decode('utf-8')\n",
    "    soup = BeautifulSoup(html, features='lxml')\n",
    "    return soup\n",
    "##爬取所有的射击h推荐游戏以及它们的超链接\n",
    "def spider_all_action(soup):\n",
    "    #第一步筛选推荐游戏的区域，在页面当中的区域为div class='wrap'\n",
    "    first_step = soup.find(lambda tag:tag.name=='div' and tag.get('class')==['wrap'])\n",
    "    #第二步先大致选出首页当中推荐的所有射击游戏\n",
    "    second_step = first_step.find('ul',{'class':'game_list'})\n",
    "    #直接取p标签，来获取游戏名称\n",
    "    print(type(second_step))\n",
    "    last_step = second_step.find_all('a')\n",
    "    #\n",
    "    for game in last_step:\n",
    "        #定义一个字典类型用于保存游戏名称以及对应的超链接\n",
    "        dic1 = {}\n",
    "        #寻找a标签的超链接href\n",
    "        href = game.get('href')\n",
    "        name = game.find('p').text\n",
    "        # print(href+name)\n",
    "        dic1[name] = href\n",
    "        all_game.append(dic1)\n",
    "    return all_game\n",
    "#爬取游戏的相关信息 例如游戏介绍 大小 发行日期\n",
    "def spider_info(info):\n",
    "    dic2 = {}\n",
    "    for i in info:\n",
    "        for key, value in i.items():\n",
    "#             if\n",
    "            if 'web.7k7k.com' not in value:\n",
    "                soup = web_content(value)\n",
    "                #soup1代表的是游戏的大小 发行日期  soup2代表的是游戏的介绍信息\n",
    "                soup1 = soup.find('p',{'class':'game_info_f1'})\n",
    "                dic_temp = {}\n",
    "                if soup1 is not None:\n",
    "                    #查找所有的子标签 span\n",
    "                    info = soup1.find_all('span')\n",
    "                    del info[0]  #删除第一个元素，因为这个不是我们想要的\n",
    "                    for i in info:\n",
    "                        i = i.text\n",
    "                        detail_info = i.split(\"：\")\n",
    "                        dic_temp[detail_info[0]] = detail_info[1]\n",
    "                    intro = soup.find('div', {'class': 'game_desc'})\n",
    "                    if intro is not None:\n",
    "                        introduce = intro.find(text=True).strip()\n",
    "                        dic_temp[\"介绍\"] = introduce\n",
    "                        dic2[key] = dic_temp\n",
    "                    print(dic_temp)\n",
    "                else:\n",
    "                    dic_temp[\"大小\"] = \"无\"\n",
    "                    dic_temp[\"日期\"] = \"无\"\n",
    "                    dic_temp[\"介绍\"] = \"无\"\n",
    "                    dic2[key] = dic_temp\n",
    "            else:\n",
    "                dic_temp[\"大小\"] = \"无\"\n",
    "                dic_temp[\"日期\"] = \"无\"\n",
    "                dic_temp[\"介绍\"] = \"无\"\n",
    "                dic2[key] = dic_temp\n",
    "    return dic2\n",
    "#开5个线程，对数据进行爬取\n",
    "def thread1():\n",
    "    for page in range(0,10):\n",
    "        url = 'http://www.7k7k.com/xiuxian/index_'+str(page+1)+'.htm'\n",
    "        info = spider_all_action(web_content(url))\n",
    "def thread2():\n",
    "    for page in range(11,20):\n",
    "        url = 'http://www.7k7k.com/xiuxian/index_'+str(page)+'.htm'\n",
    "        info = spider_all_action(web_content(url))\n",
    "def thread3():\n",
    "    for page in range(20,30):\n",
    "        url = 'http://www.7k7k.com/xiuxian/index_'+str(page)+'.htm'\n",
    "        info = spider_all_action(web_content(url))\n",
    "def thread4():\n",
    "    for page in range(30,40):\n",
    "        url = 'http://www.7k7k.com/xiuxian/index_'+str(page)+'.htm'\n",
    "        info = spider_all_action(web_content(url))\n",
    "def thread5():\n",
    "    for page in range(40,51):\n",
    "        url = 'http://www.7k7k.com/xiuxian/index_'+str(page)+'.htm'\n",
    "        info = spider_all_action(web_content(url))\n",
    "if __name__ == '__main__':\n",
    "    #定义列表 用于保存所有的射击游戏以及它们的超链接\n",
    "    all_game = []\n",
    "    info = None\n",
    "    #添加线程\n",
    "    thread01 = threading.Thread(target=thread1,name=\"T1\")\n",
    "    thread02 = threading.Thread(target=thread2,name=\"T2\")\n",
    "    thread03 = threading.Thread(target=thread3,name=\"T3\")\n",
    "    thread04 = threading.Thread(target=thread4,name=\"T4\")\n",
    "    thread05 = threading.Thread(target=thread5,name=\"T5\")\n",
    "    #线程开始执行\n",
    "    thread01.start()\n",
    "    thread02.start()\n",
    "    thread03.start()\n",
    "    thread04.start()\n",
    "    thread05.start()\n",
    "    #爬取所有的页面所推荐的射击游戏\n",
    "#     for page in range(0,10):\n",
    "#         url = 'http://www.7k7k.com/xiuxian/index_'+str(page+1)+'.htm'\n",
    "#         info = spider_all_action(web_content(url))\n",
    "#     url = 'http://www.7k7k.com/swf/200150.htm'\n",
    "#     url = 'http://www.7k7k.com/flash/197869.htm'\n",
    "#     info = spider_all_action(web_content(url))\n",
    "    d = spider_info(info)\n",
    "    with open('multiThreadxiuxian_intro.txt', 'w', encoding='utf-8') as f:\n",
    "        f.write(json.dumps(d,ensure_ascii=False))\n",
    "    print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca3de1c4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
